<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   Python机器学习——线性模型 – 郝智恒 | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script type="text/x-mathjax-config">
   MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']], processEscapes: true}});
  </script>
  <script src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML%2CSafe.js&amp;ver=15f5e1fcddf0b88517244c085cc07855" type="text/javascript">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://blog.cos.name/bigknife/2013/03/03/python%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0-%E7%BA%BF%E6%80%A7%E6%A8%A1%E5%9E%8B%EF%BC%881%EF%BC%89/",
                  host: "http://blog.cos.name",
                  prePath: "http://blog.cos.name",
                  scheme: "http",
                  pathBase: "http://blog.cos.name/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466462554: Accept with keywords: (title(1.0):Python,线性,模型, topn(0.6):感知器,项是,分类器,学术,共线性,Python,概率模型,模型,参数,文章,用户手册,笔者,方法,系数,函数,python,准则,回归,算法,实例,拟合,线性,脚本,范数,报告,数据,代码,正则,多任务,二乘).-->
 </head>
 <body class="single single-post postid-134 single-format-standard custom-background light-theme-gray-1 suffusion-custom preset-1000px device-desktop" onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    Python机器学习——线性模型 – 郝智恒
   </h1>
  </div>
  <div class="container" id="xmain">
   <div class="fix" id="wrapper">
    <div class="custom-header fix" id="header-container">
     <header class="fix" id="header">
      <h2 class="blogtitle left">
       <a href="http://blog.cos.name/bigknife">
        郝智恒
       </a>
      </h2>
      <div class="description right">
       一个机器学习菜鸟
      </div>
     </header>
     <!-- /header -->
     <nav class="tab fix" id="nav">
      <div class="col-control left">
       <!-- right-header-widgets -->
       <div class="warea" id="right-header-widgets">
        <form action="http://blog.cos.name/bigknife/" class="searchform " method="get">
        </form>
       </div>
       <!-- /right-header-widgets -->
      </div>
      <!-- /col-control -->
     </nav>
     <!-- /nav -->
    </div>
    <!-- //#header-container -->
    <div class="fix" id="container">
     <div id="main-col">
      <div id="content">
       <article class="post-134 post type-post status-publish format-standard hentry category-ml category-python tag-python- category-12-id category-11-id full-content meta-position-corners fix" id="post-134">
        <header class="post-header title-container fix">
         <div class="title">
          <h1 class="posttitle">
           <a class="entry-title" href="http://blog.cos.name/bigknife/2013/03/03/python%e6%9c%ba%e5%99%a8%e5%ad%a6%e4%b9%a0-%e7%ba%bf%e6%80%a7%e6%a8%a1%e5%9e%8b%ef%bc%881%ef%bc%89/" rel="bookmark" title="Python机器学习——线性模型">
            Python机器学习——线性模型
           </a>
          </h1>
          <div class="postdata fix">
           <span class="category">
            <span class="icon">
            </span>
            <a href="http://blog.cos.name/bigknife/category/ml/" rel="category tag">
             ML
            </a>
            ,
            <a href="http://blog.cos.name/bigknife/category/python/" rel="category tag">
             python
            </a>
           </span>
           <span class="comments">
            <span class="icon">
            </span>
            <a href="http://blog.cos.name/bigknife/2013/03/03/python%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0-%E7%BA%BF%E6%80%A7%E6%A8%A1%E5%9E%8B%EF%BC%881%EF%BC%89/#respond">
             Add comments
            </a>
           </span>
          </div>
          <!-- /.postdata -->
         </div>
         <!-- /.title -->
         <div class="date">
          <span class="month">
           3月
          </span>
          <span class="day">
           03
          </span>
          <span class="year">
           2013
          </span>
         </div>
        </header>
        <!-- /.title-container -->
        <span class="post-format-icon">
        </span>
        <span class="updated" title="2013-03-03T12:24:26+00:00">
        </span>
        <div class="entry-container fix">
         <div class="entry fix">
          <p>
           最近断断续续地在接触一些python的东西。按照我的习惯，首先从应用层面搞起，尽快入门，后续再细化一些技术细节。找了一些资料，基本语法和数据结构搞定之后，目光便转到了scikit-learn这个包。这个包是基于scipy的统计学习包。里面所涵盖的算法接口非常全面。更令人振奋的是，其用户手册写得非常好。然而，其被墙了（或者没有，或者有时被墙有时又好了）。笔者不会翻墙（请嘲笑吧），笔者只有找代理，笔者要忍受各种弹窗。因此笔者决定，做一个记录，把这用户手册的内容多多少少搬一点到我的博客中来。以备查询方便。因此，笔者就动手了。
          </p>
          <p>
           声明：如何安装python及其IDE，相关模块等等不在本系列博客的范围。本博客仅试图记录可能会用到的代码实例。
          </p>
          <h1>
           1.广义线性模型
          </h1>
          <p>
           这里的“广义线性模型”，是指线性模型及其简单的推广，包括岭回归，lasso，LAR，logistic回归，感知器等等。下面将介绍这些模型的基本想法，以及如何用python实现。
          </p>
          <h2>
           1.1.普通的最小二乘
          </h2>
          <p>
           由
           <code>
            LinearRegression
           </code>
           函数实现。最小二乘法的缺点是依赖于自变量的相关性，当出现复共线性时，设计阵会接近奇异，因此由最小二乘方法得到的结果就非常敏感，如果随机误差出现什么波动，最小二乘估计也可能出现较大的变化。而当数据是由非设计的试验获得的时候，复共线性出现的可能性非常大。
          </p>
          <pre>from sklearn import linear_model
clf = linear_model.LinearRegression()
clf.fit ([[0,0],[1,1],[2,2]],[0,1,2]) #拟合
clf.coef_ #获取拟合参数</pre>
          <p>
           上面代码给出了实现线性回归拟合以及获得拟合参数的两个主要函数。下面的脚本则给出了一个较为完整的实例。
          </p>
          <pre>脚本：
print __doc__

import pylab as pl
import numpy as np
from sklearn import datasets, linear_model

diabetes = datasets.load_diabetes() #载入数据

diabetes_x = diabetes.data[:, np.newaxis]
diabetes_x_temp = diabetes_x[:, :, 2]

diabetes_x_train = diabetes_x_temp[:-20] #训练样本
diabetes_x_test = diabetes_x_temp[-20:] #检测样本
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test = diabetes.target[-20:]

regr = linear_model.LinearRegression()

regr.fit(diabetes_x_train, diabetes_y_train)

print 'Coefficients :\n', regr.coef_

print ("Residual sum of square: %.2f" %np.mean((regr.predict(diabetes_x_test) - diabetes_y_test) ** 2))

print ("variance score: %.2f" % regr.score(diabetes_x_test, diabetes_y_test))

pl.scatter(diabetes_x_test,diabetes_y_test, color = 'black')
pl.plot(diabetes_x_test, regr.predict(diabetes_x_test),color='blue',linewidth = 3)
pl.xticks(())
pl.yticks(())
pl.show()</pre>
          <h2>
           1.2.岭回归
          </h2>
          <p>
           岭回归是一种正则化方法，通过在损失函数中加入L2范数惩罚项，来控制线性模型的复杂程度，从而使得模型更稳健。岭回归的函数如下：
          </p>
          <pre>from sklearn import linear_model
clf = linear_model.Ridge (alpha = .5)
clf.fit([[0,0],[0,0],[1,1]],[0,.1,1])
clf.coef_</pre>
          <p>
           下面的脚本提供了绘制岭估计系数与惩罚系数之间关系的功能
          </p>
          <pre>print __doc__

import numpy as np
import pylab as pl
from sklearn import linear_model

# X is the 10x10 Hilbert matrix
X = 1. / (np.arange(1, 11) + np.arange(0, 10)[:, np.newaxis])
y = np.ones(10)

# Compute paths
n_alphas = 200
alphas = np.logspace(-10, -2, n_alphas)
clf = linear_model.Ridge(fit_intercept=False) #创建一个岭回归对象

coefs = []#循环，对每一个alpha，做一次拟合
for a in alphas:
clf.set_params(alpha=a)
clf.fit(X, y)
coefs.append(clf.coef_)#系数保存在coefs中，append

# Display results
ax = pl.gca()
ax.set_color_cycle(['b', 'r', 'g', 'c', 'k', 'y', 'm'])
ax.plot(alphas, coefs)
ax.set_xscale('log') #注意这一步，alpha是对数化了的
ax.set_xlim(ax.get_xlim()[::-1]) # reverse axis
pl.xlabel('alpha')
pl.ylabel('weights')
pl.title('Ridge coefficients as a function of the regularization')
pl.axis('tight')
pl.show()</pre>
          <p>
           使用GCV来设定正则化系数的代码如下：
          </p>
          <pre>clf = linear_model.RidgeCV(alpha = [0.1, 1.0, 10.0])
clf.fit([[0,0],[0,0],[1,1]],[0,.1,1])
clf.alpha_</pre>
          <h2>
           1.3. Lasso
          </h2>
          <p>
           lasso和岭估计的区别在于它的惩罚项是基于L1范数的。因此，它可以将系数控制收缩到0，从而达到变量选择的效果。它是一种非常流行的变量选择方法。Lasso估计的算法主要有两种，其一是用于以下介绍的函数Lasso的coordinate descent。另外一种则是下面会介绍到的最小角回归（笔者学生阶段读过的最令人佩服的文章之一便是Efron的这篇LARS，完全醍醐灌顶，建议所有人都去读一读）。
          </p>
          <pre>clf = linear_model.Lasso(alpha = 0.1)
clf.fit([[0,0],[1,1]],[0,1])
clf.predict([[1,1]])</pre>
          <p>
           下面给出一个脚本，比较了Lasso和Elastic Net在处理稀疏信号中的应用
          </p>
          <pre>print __doc__

import numpy as np
import pylab as pl

from sklearn.metrics import r2_score


# generate some sparse data to play with
np.random.seed(42)

n_samples, n_features = 50, 200
X = np.random.randn(n_samples, n_features)
coef = 3 * np.random.randn(n_features)
inds = np.arange(n_features)
np.random.shuffle(inds)#打乱观测顺序
coef[inds[10:]] = 0 # sparsify coef
y = np.dot(X, coef)

# add noise
y += 0.01 * np.random.normal((n_samples,))

# Split data in train set and test set
n_samples = X.shape[0]
X_train, y_train = X[:n_samples / 2], y[:n_samples / 2]
X_test, y_test = X[n_samples / 2:], y[n_samples / 2:]

# Lasso
from sklearn.linear_model import Lasso

alpha = 0.1
lasso = Lasso(alpha=alpha)#Lasso对象

y_pred_lasso = lasso.fit(X_train, y_train).predict(X_test)#拟合并预测
r2_score_lasso = r2_score(y_test, y_pred_lasso)
print lasso
print "r^2 on test data : %f" % r2_score_lasso

# ElasticNet
from sklearn.linear_model import ElasticNet
enet = ElasticNet(alpha=alpha, l1_ratio=0.7)
y_pred_enet = enet.fit(X_train, y_train).predict(X_test)
r2_score_enet = r2_score(y_test, y_pred_enet)
print enet
print "r^2 on test data : %f" % r2_score_enet

pl.plot(enet.coef_, label='Elastic net coefficients')
pl.plot(lasso.coef_, label='Lasso coefficients')
pl.plot(coef, '--', label='original coefficients')
pl.legend(loc='best')
pl.title("Lasso R^2: %f, Elastic Net R^2: %f"
% (r2_score_lasso, r2_score_enet))
pl.show()</pre>
          <h3>
           1.3.1.如何设置正则化系数
          </h3>
          <h4>
           1.3.1.1. 使用交叉验证
          </h4>
          <p>
           有两个函数实现了交叉验证，
           <code>
            LassoCV
           </code>
           ，另一个是
           <code>
            LassoLarsCV
           </code>
           。对于高维数据而言，选择
           <code>
            LassoCV
           </code>
           往往是最适合的方法，然而
           <code>
            LassoLarsCV
           </code>
           速度要快一些。
          </p>
          <h4>
           1.3.1.2. 使用信息准则
          </h4>
          <p>
           AIC,BIC。这些准则计算起来比cross validation方法消耗低。然而使用这些准则的前提是我们对模型的自由度有一个恰当的估计，并且假设我们的概率模型是正确的。事实上我们也经常遇到这种问题，我们还是更希望能直接从数据中算出些什么，而不是首先建立概率模型的假设。
          </p>
          <p>
           下面的这个脚本比较了几种设定正则化参数的方法：AIC,BIC以及cross-validation
          </p>
          <pre>import time

import numpy as np
import pylab as pl

from sklearn.linear_model import LassoCV, LassoLarsCV, LassoLarsIC
from sklearn import datasets

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

rng = np.random.RandomState(42)
X = np.c_[X, rng.randn(X.shape[0], 14)] # add some bad features

# normalize data as done by Lars to allow for comparison
X /= np.sqrt(np.sum(X ** 2, axis=0))

# LassoLarsIC: least angle regression with BIC/AIC criterion
model_bic = LassoLarsIC(criterion='bic')#BIC准则
t1 = time.time()
model_bic.fit(X, y)
t_bic = time.time() - t1
alpha_bic_ = model_bic.alpha_

model_aic = LassoLarsIC(criterion='aic')#AIC准则
model_aic.fit(X, y)
alpha_aic_ = model_aic.alpha_

def plot_ic_criterion(model, name, color):
    alpha_ = model.alpha_
    alphas_ = model.alphas_
    criterion_ = model.criterion_
    pl.plot(-np.log10(alphas_), criterion_, '--', color=color,
        linewidth=3, label='%s criterion' % name)
    pl.axvline(-np.log10(alpha_), color=color, linewidth=3,
        label='alpha: %s estimate' % name)
    pl.xlabel('-log(alpha)')
    pl.ylabel('criterion')

pl.figure()
plot_ic_criterion(model_aic, 'AIC', 'b')
plot_ic_criterion(model_bic, 'BIC', 'r')
pl.legend()
pl.title('Information-criterion for model selection (training time %.3fs)'
% t_bic)

# LassoCV: coordinate descent
# Compute paths
print "Computing regularization path using the coordinate descent lasso..."
t1 = time.time()
model = LassoCV(cv=20).fit(X, y)#创建对像，并拟合
t_lasso_cv = time.time() - t1

# Display results
m_log_alphas = -np.log10(model.alphas_)

pl.figure()
ymin, ymax = 2300, 3800
pl.plot(m_log_alphas, model.mse_path_, ':')
pl.plot(m_log_alphas, model.mse_path_.mean(axis=-1), 'k',
label='Average across the folds', linewidth=2)
pl.axvline(-np.log10(model.alpha_), linestyle='--', color='k',
label='alpha: CV estimate')

pl.legend()
pl.xlabel('-log(alpha)')
pl.ylabel('Mean square error')
pl.title('Mean square error on each fold: coordinate descent '
'(train time: %.2fs)' % t_lasso_cv)
pl.axis('tight')
pl.ylim(ymin, ymax)

# LassoLarsCV: least angle regression
# Compute paths
print "Computing regularization path using the Lars lasso..."
t1 = time.time()
model = LassoLarsCV(cv=20).fit(X, y)
t_lasso_lars_cv = time.time() - t1

# Display results
m_log_alphas = -np.log10(model.cv_alphas_)

pl.figure()
pl.plot(m_log_alphas, model.cv_mse_path_, ':')
pl.plot(m_log_alphas, model.cv_mse_path_.mean(axis=-1), 'k',
label='Average across the folds', linewidth=2)
pl.axvline(-np.log10(model.alpha_), linestyle='--', color='k',
label='alpha CV')
pl.legend()
pl.xlabel('-log(alpha)')
pl.ylabel('Mean square error')
pl.title('Mean square error on each fold: Lars (train time: %.2fs)'
% t_lasso_lars_cv)
pl.axis('tight')
pl.ylim(ymin, ymax)
pl.show()</pre>
          <h2>
           1.4. Elastic Net
          </h2>
          <p>
           ElasticNet是对Lasso和岭回归的融合，其惩罚项是L1范数和L2范数的一个权衡。下面的脚本比较了Lasso和Elastic Net的回归路径，并做出了其图形。
          </p>
          <pre>print __doc__

# Author: Alexandre Gramfort &lt;alexandre.gramfort@inria.fr&gt;
# License: BSD Style.

import numpy as np
import pylab as pl

from sklearn.linear_model import lasso_path, enet_path
from sklearn import datasets

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

X /= X.std(0)  # Standardize data (easier to set the l1_ratio parameter)

# Compute paths

eps = 5e-3  # the smaller it is the longer is the path

print "Computing regularization path using the lasso..."
models = lasso_path(X, y, eps=eps)
alphas_lasso = np.array([model.alpha for model in models])
coefs_lasso = np.array([model.coef_ for model in models])

print "Computing regularization path using the positive lasso..."
models = lasso_path(X, y, eps=eps, positive=True)#lasso path
alphas_positive_lasso = np.array([model.alpha for model in models])
coefs_positive_lasso = np.array([model.coef_ for model in models])

print "Computing regularization path using the elastic net..."
models = enet_path(X, y, eps=eps, l1_ratio=0.8)
alphas_enet = np.array([model.alpha for model in models])
coefs_enet = np.array([model.coef_ for model in models])

print "Computing regularization path using the positve elastic net..."
models = enet_path(X, y, eps=eps, l1_ratio=0.8, positive=True)
alphas_positive_enet = np.array([model.alpha for model in models])
coefs_positive_enet = np.array([model.coef_ for model in models])

# Display results

pl.figure(1)
ax = pl.gca()
ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k'])
l1 = pl.plot(coefs_lasso)
l2 = pl.plot(coefs_enet, linestyle='--')

pl.xlabel('-Log(lambda)')
pl.ylabel('weights')
pl.title('Lasso and Elastic-Net Paths')
pl.legend((l1[-1], l2[-1]), ('Lasso', 'Elastic-Net'), loc='lower left')
pl.axis('tight')

pl.figure(2)
ax = pl.gca()
ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k'])
l1 = pl.plot(coefs_lasso)
l2 = pl.plot(coefs_positive_lasso, linestyle='--')

pl.xlabel('-Log(lambda)')
pl.ylabel('weights')
pl.title('Lasso and positive Lasso')
pl.legend((l1[-1], l2[-1]), ('Lasso', 'positive Lasso'), loc='lower left')
pl.axis('tight')

pl.figure(3)
ax = pl.gca()
ax.set_color_cycle(2 * ['b', 'r', 'g', 'c', 'k'])
l1 = pl.plot(coefs_enet)
l2 = pl.plot(coefs_positive_enet, linestyle='--')

pl.xlabel('-Log(lambda)')
pl.ylabel('weights')
pl.title('Elastic-Net and positive Elastic-Net')
pl.legend((l1[-1], l2[-1]), ('Elastic-Net', 'positive Elastic-Net'),
          loc='lower left')
pl.axis('tight')
pl.show()</pre>
          <h2>
           1.5. 多任务Lasso
          </h2>
          <p>
           多任务Lasso其实就是多元Lasso。Lasso在多元回归中的推广的tricky在于如何设置惩罚项。这里略。
          </p>
          <h2>
           1.6. 最小角回归
          </h2>
          <p>
           <code>
            LassoLars
           </code>
           给出了LARS算法求解Lasso的接口。下面的脚本给出了如何用LARS画出Lasso的path。
          </p>
          <pre>print __doc__

# Author: Fabian Pedregosa &lt;fabian.pedregosa@inria.fr&gt;
#         Alexandre Gramfort &lt;alexandre.gramfort@inria.fr&gt;
# License: BSD Style.

import numpy as np
import pylab as pl

from sklearn import linear_model
from sklearn import datasets

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

print "Computing regularization path using the LARS ..."
alphas, _, coefs = linear_model.lars_path(X, y, method='lasso', verbose=True)#lars算法的求解路径

xx = np.sum(np.abs(coefs.T), axis=1)
xx /= xx[-1]

pl.plot(xx, coefs.T)
ymin, ymax = pl.ylim()
pl.vlines(xx, ymin, ymax, linestyle='dashed')
pl.xlabel('|coef| / max|coef|')
pl.ylabel('Coefficients')
pl.title('LASSO Path')
pl.axis('tight')
pl.show()</pre>
          <h2>
           logistic 回归
          </h2>
          <p>
           Logistic回归是一个线性分类器。类
           <code>
            LogisticRegression
           </code>
           实现了该分类器，并且实现了L1范数，L2范数惩罚项的logistic回归。下面的脚本是一个例子，将8*8像素的数字图像分成了两类，其中0-4分为一类，5-9分为一类。比较了L1，L2范数惩罚项，在不同的C值的情况。
           <br/>
           脚本如下:
          </p>
          <pre>print __doc__

# Authors: Alexandre Gramfort 
#          Mathieu Blondel 
#          Andreas Mueller 
# License: BSD Style.

import numpy as np
import pylab as pl

from sklearn.linear_model import LogisticRegression
from sklearn import datasets
from sklearn.preprocessing import StandardScaler

digits = datasets.load_digits()

X, y = digits.data, digits.target
X = StandardScaler().fit_transform(X)

# classify small against large digits
y = (y &gt; 4).astype(np.int)


# Set regularization parameter
for i, C in enumerate(10. ** np.arange(1, 4)):
    # turn down tolerance for short training time
    clf_l1_LR = LogisticRegression(C=C, penalty='l1', tol=0.01)
    clf_l2_LR = LogisticRegression(C=C, penalty='l2', tol=0.01)
    clf_l1_LR.fit(X, y)
    clf_l2_LR.fit(X, y)

    coef_l1_LR = clf_l1_LR.coef_.ravel()
    coef_l2_LR = clf_l2_LR.coef_.ravel()

    # coef_l1_LR contains zeros due to the
    # L1 sparsity inducing norm

    sparsity_l1_LR = np.mean(coef_l1_LR == 0) * 100
    sparsity_l2_LR = np.mean(coef_l2_LR == 0) * 100

    print "C=%d" % C
    print "Sparsity with L1 penalty: %.2f%%" % sparsity_l1_LR
    print "score with L1 penalty: %.4f" % clf_l1_LR.score(X, y)
    print "Sparsity with L2 penalty: %.2f%%" % sparsity_l2_LR
    print "score with L2 penalty: %.4f" % clf_l2_LR.score(X, y)

    l1_plot = pl.subplot(3, 2, 2 * i + 1)
    l2_plot = pl.subplot(3, 2, 2 * (i + 1))
    if i == 0:
        l1_plot.set_title("L1 penalty")
        l2_plot.set_title("L2 penalty")

    l1_plot.imshow(np.abs(coef_l1_LR.reshape(8, 8)), interpolation='nearest',
                   cmap='binary', vmax=1, vmin=0)
    l2_plot.imshow(np.abs(coef_l2_LR.reshape(8, 8)), interpolation='nearest',
                   cmap='binary', vmax=1, vmin=0)
    pl.text(-8, 3, "C = %d" % C)

    l1_plot.set_xticks(())
    l1_plot.set_yticks(())
    l2_plot.set_xticks(())
    l2_plot.set_yticks(())

pl.show()
</pre>
          <h2>
           其他
          </h2>
          <p>
           官方用户指南中还包含了感知器，Passive Aggressive算法等。本文略去。
          </p>
         </div>
         <!--/entry -->
        </div>
        <!-- .entry-container -->
        <footer class="post-footer postdata fix">
         <span class="author">
          <span class="icon">
          </span>
          Posted by
          <span class="vcard">
           <a class="url fn" href="http://blog.cos.name/bigknife/author/bigknife/" rel="author">
            bigknife
           </a>
          </span>
          at 12:24
         </span>
         <span class="tags tax">
          <span class="icon">
          </span>
          Tagged with:
          <a href="http://blog.cos.name/bigknife/tag/python-%e6%9c%ba%e5%99%a8%e5%ad%a6%e4%b9%a0/" rel="tag">
           python 机器学习
          </a>
         </span>
        </footer>
        <!-- .post-footer -->
        <section id="comments">
         <h3 class="comments">
          <span class="icon">
          </span>
          One Response to “Python机器学习——线性模型”
         </h3>
         <ol class="commentlist">
          <li class="comment even thread-even depth-1 plain-nested" id="comment-21">
           <div class="comment-body" id="div-comment-21">
            <div class="comment-author fix vcard">
             <div class="comment-author-link">
              <cite class="fn">
               coucar
              </cite>
              <span class="says">
               says:
              </span>
             </div>
             <div class="comment-meta commentmetadata">
              <a href="http://blog.cos.name/bigknife/2013/03/03/python%e6%9c%ba%e5%99%a8%e5%ad%a6%e4%b9%a0-%e7%ba%bf%e6%80%a7%e6%a8%a1%e5%9e%8b%ef%bc%881%ef%bc%89/#comment-21">
               2013/08/01 at 01:18
              </a>
             </div>
            </div>
            <p>
             scikit-learn的官网我也登录不了，能够ping通，但网页打不开。你用的什么在线代理？
            </p>
           </div>
           <div class="reply">
            <a aria-label="回复给coucar" class="comment-reply-link" href="http://blog.cos.name/bigknife/2013/03/03/python%e6%9c%ba%e5%99%a8%e5%ad%a6%e4%b9%a0-%e7%ba%bf%e6%80%a7%e6%a8%a1%e5%9e%8b%ef%bc%881%ef%bc%89/?replytocom=21#respond" onclick='return addComment.moveForm( "comment-21", "21", "respond", "134" )' rel="nofollow">
             回复
            </a>
           </div>
          </li>
          <!-- #comment-## -->
         </ol>
         <div class="navigation fix">
          <div class="alignleft">
          </div>
          <div class="alignright">
          </div>
         </div>
         <div class="comment-respond" id="respond">
          <h3 class="comment-reply-title" id="reply-title">
           <span class="icon">
           </span>
           Leave a Reply
           <small>
            <a href="http://blog.cos.name/bigknife/2013/03/03/python%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0-%E7%BA%BF%E6%80%A7%E6%A8%A1%E5%9E%8B%EF%BC%881%EF%BC%89/#respond" id="cancel-comment-reply-link" rel="nofollow" style="display:none;">
             Cancel reply
            </a>
           </small>
          </h3>
          <form action="http://blog.cos.name/bigknife/wp-comments-post.php" class="comment-form" id="commentform" method="post">
           <span>
           </span>
           <p>
            <label class="textarea suf-comment-label" for="comment">
             Your Comment
            </label>
            <textarea class="textarea" cols="60" id="comment" name="comment" rows="10" tabindex="4">
            </textarea>
           </p>
           <p class="form-allowed-tags">
            You may use these
            <abbr title="HyperText Markup Language">
             HTML
            </abbr>
            tags and attributes:
            <code>
             &lt;a href="" title=""&gt; &lt;abbr title=""&gt; &lt;acronym title=""&gt; &lt;b&gt; &lt;blockquote cite=""&gt; &lt;cite&gt; &lt;code&gt; &lt;del datetime=""&gt; &lt;em&gt; &lt;i&gt; &lt;q cite=""&gt; &lt;s&gt; &lt;strike&gt; &lt;strong&gt;
            </code>
           </p>
           <p>
            <label class="suf-comment-label" for="author">
             Name
            </label>
            (required)
           </p>
           <p>
            <label class="suf-comment-label" for="email">
             E-mail
            </label>
            (required)
           </p>
           <p>
            <label class="suf-comment-label" for="url">
             URI
            </label>
           </p>
           <p class="form-submit">
           </p>
           <p style="display: none;">
           </p>
           <p style="display: none;">
           </p>
          </form>
         </div>
         <!-- #respond -->
        </section>
        <!-- #comments -->
       </article>
       <!--/post -->
      </div>
      <!-- content -->
     </div>
     <!-- main col -->
     <div class="sidebar-shell sidebar-shell-right" id="sidebar-shell-1">
      <div class="dbx-group right boxed warea" id="sidebar">
       <!--widget start -->
       <aside class="dbx-box suf-widget search" id="search-2">
        <div class="dbx-content">
         <form action="http://blog.cos.name/bigknife/" class="searchform " method="get">
         </form>
        </div>
       </aside>
       <!--widget end -->
       <!--widget start -->
       <aside class="dbx-box suf-widget widget_recent_entries" id="recent-posts-2">
        <div class="dbx-content">
         <h3 class="dbx-handle plain">
          近期文章
         </h3>
         <ul>
          <li>
           <a href="http://blog.cos.name/bigknife/2013/03/03/python%e6%9c%ba%e5%99%a8%e5%ad%a6%e4%b9%a0-%e7%ba%bf%e6%80%a7%e6%a8%a1%e5%9e%8b%ef%bc%881%ef%bc%89/">
            Python机器学习——线性模型
           </a>
          </li>
         </ul>
        </div>
       </aside>
       <!--widget end -->
       <!--widget start -->
       <aside class="dbx-box suf-widget widget_recent_comments" id="recent-comments-2">
        <div class="dbx-content">
         <h3 class="dbx-handle plain">
          近期评论
         </h3>
         <ul id="recentcomments">
          <li class="recentcomments">
           <span class="comment-author-link">
            coucar
           </span>
           发表在《
           <a href="http://blog.cos.name/bigknife/2013/03/03/python%e6%9c%ba%e5%99%a8%e5%ad%a6%e4%b9%a0-%e7%ba%bf%e6%80%a7%e6%a8%a1%e5%9e%8b%ef%bc%881%ef%bc%89/#comment-21">
            Python机器学习——线性模型
           </a>
           》
          </li>
          <li class="recentcomments">
           <span class="comment-author-link">
            bigknife
           </span>
           发表在《
           <a href="http://blog.cos.name/bigknife/%e4%b8%80%e4%ba%9b%e5%ad%a6%e6%9c%af%e7%9b%b8%e5%85%b3%e7%9a%84%e6%8a%a5%e5%91%8a-%e6%8c%81%e7%bb%ad%e6%9b%b4%e6%96%b0%e4%b8%ad/#comment-12">
            一些学术相关的报告——持续更新中
           </a>
           》
          </li>
          <li class="recentcomments">
           <span class="comment-author-link">
            bigknife
           </span>
           发表在《
           <a href="http://blog.cos.name/bigknife/%e4%b8%80%e4%ba%9b%e5%ad%a6%e6%9c%af%e7%9b%b8%e5%85%b3%e7%9a%84%e6%8a%a5%e5%91%8a-%e6%8c%81%e7%bb%ad%e6%9b%b4%e6%96%b0%e4%b8%ad/#comment-11">
            一些学术相关的报告——持续更新中
           </a>
           》
          </li>
          <li class="recentcomments">
           <span class="comment-author-link">
            HaveF
           </span>
           发表在《
           <a href="http://blog.cos.name/bigknife/%e4%b8%80%e4%ba%9b%e5%ad%a6%e6%9c%af%e7%9b%b8%e5%85%b3%e7%9a%84%e6%8a%a5%e5%91%8a-%e6%8c%81%e7%bb%ad%e6%9b%b4%e6%96%b0%e4%b8%ad/#comment-10">
            一些学术相关的报告——持续更新中
           </a>
           》
          </li>
         </ul>
        </div>
       </aside>
       <!--widget end -->
       <!--widget start -->
       <aside class="dbx-box suf-widget widget_archive" id="archives-2">
        <div class="dbx-content">
         <h3 class="dbx-handle plain">
          文章归档
         </h3>
         <ul>
          <li>
           <a href="http://blog.cos.name/bigknife/2013/03/">
            2013年三月
           </a>
          </li>
         </ul>
        </div>
       </aside>
       <!--widget end -->
       <!--widget start -->
       <aside class="dbx-box suf-widget widget_categories" id="categories-2">
        <div class="dbx-content">
         <h3 class="dbx-handle plain">
          分类目录
         </h3>
         <ul>
          <li class="cat-item cat-item-12">
           <a href="http://blog.cos.name/bigknife/category/ml/">
            ML
           </a>
          </li>
          <li class="cat-item cat-item-11">
           <a href="http://blog.cos.name/bigknife/category/python/">
            python
           </a>
          </li>
         </ul>
        </div>
       </aside>
       <!--widget end -->
       <!--widget start -->
       <aside class="dbx-box suf-widget widget_meta" id="meta-2">
        <div class="dbx-content">
         <h3 class="dbx-handle plain">
          功能
         </h3>
         <ul>
          <li>
           <a href="http://blog.cos.name/bigknife/wp-login.php">
            登录
           </a>
          </li>
          <li>
           <a href="http://blog.cos.name/bigknife/feed/">
            文章
            <abbr title="Really Simple Syndication">
             RSS
            </abbr>
           </a>
          </li>
          <li>
           <a href="http://blog.cos.name/bigknife/comments/feed/">
            评论
            <abbr title="Really Simple Syndication">
             RSS
            </abbr>
           </a>
          </li>
          <li>
           <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
            WordPress.org
           </a>
          </li>
         </ul>
        </div>
       </aside>
       <!--widget end -->
       <!--widget start -->
       <aside class="dbx-box suf-widget widget_links" id="linkcat-2">
        <div class="dbx-content">
         <h3 class="dbx-handle plain">
          Blogroll
         </h3>
         <ul class="xoxo blogroll">
          <li>
           <a href="http://yihui.name/">
            yihui
           </a>
           神
          </li>
          <li>
           <a href="http://www.road2stat.com/">
            肖楠
           </a>
          </li>
         </ul>
        </div>
       </aside>
       <!--widget end -->
      </div>
      <!--/sidebar -->
      <!-- #sidebar-b -->
      <div class="dbx-group right boxed warea" id="sidebar-b">
       <!--widget start -->
       <aside class="dbx-box suf-widget widget_tag_cloud" id="tag_cloud-2">
        <div class="dbx-content">
         <h3 class="dbx-handle plain">
          标签
         </h3>
         <div class="tagcloud">
          <a class="tag-link-14 tag-link-position-1" href="http://blog.cos.name/bigknife/tag/python-%e6%9c%ba%e5%99%a8%e5%ad%a6%e4%b9%a0/" style="font-size: 8pt;" title="1个话题">
           python 机器学习
          </a>
         </div>
        </div>
       </aside>
       <!--widget end -->
      </div>
      <!-- /#sidebar-b -->
     </div>
    </div>
    <!-- /container -->
    <footer>
     <div id="cred">
      <table>
       <tbody>
        <tr>
         <td class="cred-left">
          © 2013
          <a href="http://blog.cos.name/bigknife">
           郝智恒
          </a>
         </td>
         <td class="cred-center">
         </td>
         <td class="cred-right">
          <a href="http://aquoid.com/news/themes/suffusion/">
           Suffusion theme by Sayontan Sinha
          </a>
         </td>
        </tr>
       </tbody>
      </table>
     </div>
    </footer>
    <!-- 57 queries, 48MB in 0.997 seconds. -->
   </div>
   <!--/wrapper -->
   <!-- location footer -->
  </div>
 </body>
</html>